<!DOCTYPE html>



  


<html class="theme-next gemini use-motion" lang="zh-CN">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>
<meta name="theme-color" content="#222">









<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />















  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  
    
      
    

    
  

  
    
      
    

    
  

  
    
      
    

    
  

  
    
      
    

    
  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Microsoft YaHei:300,300italic,400,400italic,700,700italic|Microsoft YaHei:300,300italic,400,400italic,700,700italic|Microsoft YaHei:300,300italic,400,400italic,700,700italic|Microsoft YaHei:300,300italic,400,400italic,700,700italic|Inziu Iosevka Slab SC:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.2" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="Paper Reading," />








  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=5.1.2" />






<meta name="description" content="However, much of the “ folk knowledge ” that is needed to successfully develop machine learning applications is not readily available in them. As a result, many machine learning pro jects take much lo">
<meta name="keywords" content="Paper Reading">
<meta property="og:type" content="article">
<meta property="og:title" content="A Few Useful Things to Know About Machine Learning">
<meta property="og:url" content="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/index.html">
<meta property="og:site_name" content="苦舟">
<meta property="og:description" content="However, much of the “ folk knowledge ” that is needed to successfully develop machine learning applications is not readily available in them. As a result, many machine learning pro jects take much lo">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/markdown-img-paste-20170923091337661.png">
<meta property="og:image" content="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/markdown-img-paste-20170922163904996.png">
<meta property="og:updated_time" content="2017-11-22T15:33:53.500Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="A Few Useful Things to Know About Machine Learning">
<meta name="twitter:description" content="However, much of the “ folk knowledge ” that is needed to successfully develop machine learning applications is not readily available in them. As a result, many machine learning pro jects take much lo">
<meta name="twitter:image" content="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/markdown-img-paste-20170923091337661.png">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    sidebar: {"position":"left","display":"hide","offset":12,"offset_float":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: true,
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/"/>





  <title>A Few Useful Things to Know About Machine Learning | 苦舟</title>
  














</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/"  class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">苦舟</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle">学海无涯，吾将上下求索。</p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br />
            
            分类
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-commonweal">
          <a href="/404.html" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-heartbeat"></i> <br />
            
            公益404
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            搜索
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup search-popup local-search-popup">
  <div class="local-search-header clearfix">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
    <div class="local-search-input-wrapper">
      <input autocomplete="off"
             placeholder="搜索..." spellcheck="false"
             type="text" id="local-search-input">
    </div>
  </div>
  <div id="local-search-result"></div>
</div>



    </div>
  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://idmk.oschina.io/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="东木金">
      <meta itemprop="description" content="">
      <meta itemprop="image" content="/uploads/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="苦舟">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">A Few Useful Things to Know About Machine Learning</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2017-09-22T09:07:02+08:00">
                2017-09-22
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/ML/" itemprop="url" rel="index">
                    <span itemprop="name">ML</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <p>However, much of the “ folk knowledge ” that is needed to successfully develop machine learning applications is not readily available in them. As a result, many machine learning pro jects take much longer than necessary or wind up producing lessthan-ideal results. Yet much of this folk knowledge is fairly easy to communicate. This is the purpose of this article.</p>
<a id="more"></a>
<h2 id="Learning-Representation-Evaluation-optimization-学习-表示-评价-优化"><a href="#Learning-Representation-Evaluation-optimization-学习-表示-评价-优化" class="headerlink" title="Learning = Representation + Evaluation + optimization 学习 = 表示 + 评价 + 优化"></a>Learning = Representation + Evaluation + optimization 学习 = 表示 + 评价 + 优化</h2><p>这个在前面的 Andrew 的课中就可以找到例子。逻辑回归是一种模型，就是这里的“表示”；成本函数用的是似然估计，就是这里的“评价”；优化用的是梯度下降。这样三个部分组成的是完整的机器学习的算法。</p>
<h2 id="It-is-Generalization-that-Counts-泛化（Generalization）很重要"><a href="#It-is-Generalization-that-Counts-泛化（Generalization）很重要" class="headerlink" title="It is Generalization that Counts 泛化（Generalization）很重要"></a>It is Generalization that Counts 泛化（Generalization）很重要</h2><p>Generalization 泛化就是一般化。泛化能力是指模型处理新数据的能力。我们都是用训练数据来学习，但是机器学习的目的都是为了预测新的没见过的数据，而不是已经知道答案的训练数据。所以，如果用训练数据来测试，当然结果很好，但是这其实并不正确。</p>
<p>可以怎么做呢？藏一些数据。（交叉验证）比如，把数据分成 10 份，9 份当训练数据，1 份来做测试。</p>
<h2 id="Data-Alone-is-not-Enough-仅有数据还不够"><a href="#Data-Alone-is-not-Enough-仅有数据还不够" class="headerlink" title="Data Alone is not Enough 仅有数据还不够"></a>Data Alone is not Enough 仅有数据还不够</h2><p>样本空间的大小大于训练样本的数量。<br>在 Hsuan-Tien Lin 的课上，说到学习是可行的时候，见过假设集合 hypothesis set 是比数据还要多的。N 笔（xi,yi）的数据，可能有 h=2^N，虽然，后来证明其实没有这么多，但是也是比数据要多（Hsuan-Tien Lin 第五、六讲中有证明）。既然，假设的集合那么多，为什么机器学习又能做的好呢？作者解释到，真正的机器学习并不是从假设集合中一个一个挑，而是有一些先验知识帮助我们更好的筛选 h。先验知识就是，针对不同领域，已经拥有的可以使机器学习更容易做出选择，选到好的 h 的知识。比如，如果概率很牛，那么用图模型（把概率分布中的条件，独立用图的形式表达出来）就稍微容易了。</p>
<p>如果没有这样的经验，就让机器自己去茫茫假设集合中选，还能挑到好的，还不如抛个硬币更容易。所以没有“经验 +data ”，就做选择，会说一句“ no free lunch ”来告诫。有了经验，就好像杠杆一样，选到靠前的支点，就能用更少的知识获得更好的效果。</p>
<h2 id="Overfitting-has-many-Faces-过拟合（Overfitting）有多张面孔"><a href="#Overfitting-has-many-Faces-过拟合（Overfitting）有多张面孔" class="headerlink" title="Overfitting has many Faces 过拟合（Overfitting）有多张面孔"></a>Overfitting has many Faces 过拟合（Overfitting）有多张面孔</h2><p>过拟合在 Andrew Week3 提到过。过拟合就是在训练数据上表现太好，而在新数据上表现的很糟糕。上面讲的泛化是目标，那过拟合就是要强化避免的。造成过拟合的原因是训练数据太少或者模型太复杂（表现出来的就是特征很多）。</p>
<p>泛化误差有两个方面造成，bias 和 variance。用投飞镖的例子来看，靶心就是好的 h。上面的点 X 表示一次对一个数据集 D，学习后得到的 h。可以看出，高 bias 表示离中心越远，高 variance 表示对不同数据集学习得到的 h 很分散。<br><img src="/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/markdown-img-paste-20170923091337661.png" alt="markdown-img-paste-20170923091337661.png" title=""><br>in-sample error 就是模型在训练数据中的表现的误差，误差越大就是越 higher bias。<br>out-of-sample error 就是模型对新数据的表现的误差，误差越大就是越 higher vanriance。<br>从上还能看出复杂的模型有较大 vanriance 较小的 bias。而模型和算法都是由特征选择产生的，这也就是在 Andrew 课中说到的，为什么 variance 高时要惩罚一些特征。</p>
<h2 id="Intuition-Fails-in-high-Dimensions-直觉不适用于高维空间"><a href="#Intuition-Fails-in-high-Dimensions-直觉不适用于高维空间" class="headerlink" title="Intuition Fails in high Dimensions 直觉不适用于高维空间"></a>Intuition Fails in high Dimensions 直觉不适用于高维空间</h2><p>维度灾难。</p>
<h2 id="theoretical-Guarantees-Are-NotWhat-they-Seem-理论保证（Theoretical-Guarantees）与看上去的不一样"><a href="#theoretical-Guarantees-Are-NotWhat-they-Seem-理论保证（Theoretical-Guarantees）与看上去的不一样" class="headerlink" title="theoretical Guarantees Are NotWhat they Seem 理论保证（Theoretical  Guarantees）与看上去的不一样"></a>theoretical Guarantees Are NotWhat they Seem 理论保证（Theoretical  Guarantees）与看上去的不一样</h2><p>这里提了两个保证：边界保证和渐进保证。<br>边界的意思是，给定一个足够大的训练集，告诉你在很大的概率上你的学习器会返回一个成功泛化的假设，还是无法找到一个保持正确的假设。<br>渐进保证是，给定无穷数据，学习器将保证输出正确的分类器。（来自译文翻译）<br>边界保证其实在 Hsuan-Tien Lin 的课上讲到（第四、五讲，可以去看看）过，是为了解决机器学习可行性问题。<br>渐进保证，是很少会处于的一种状况。<br>文章说到，理论保证就是为了理解，不会再实际运用中起决策作用，顶多就是在设计算法的时候给些提示。</p>
<h2 id="Feature-Engineering-Is-the-Key-特征工程（Feature-Engineering）是关键"><a href="#Feature-Engineering-Is-the-Key-特征工程（Feature-Engineering）是关键" class="headerlink" title="Feature Engineering Is the Key 特征工程（Feature Engineering）是关键"></a>Feature Engineering Is the Key 特征工程（Feature Engineering）是关键</h2><p>特征的确是机器学习成败的关键，难怪现在这么多搞 Deep learning 的。我们在用机器学习做预测的时候，发现真正学习的时候并不多，看看数据的收集、整理、清理和预处理，还有就是特征的设计，试验，修正等。且，机器学习是无法一次就达到的，需要进行多次迭代。特征工程之所以比学习还要复杂，是因为它是和领域相关的，机器学习算法是通用的。比如，银行预测是否给顾客信用卡，你用二元分类，但是特征就是银行顾客的资料。比如病人是否有癌症，你也用二元分类，但是特征就是病人的资料。这些领域知识，在“仅有数据是不够的”那节说到，需要人们的先验知识，但是，我们还是希望机器学习越来越智能。</p>
<h2 id="more-Data-Beats-a-Cleverer-Algorithm-更多是数据胜过更聪明的算法"><a href="#more-Data-Beats-a-Cleverer-Algorithm-更多是数据胜过更聪明的算法" class="headerlink" title="more Data Beats a Cleverer Algorithm 更多是数据胜过更聪明的算法"></a>more Data Beats a Cleverer Algorithm 更多是数据胜过更聪明的算法</h2><p>这节说到，当遇到分类器效果不够好的时候，有两个选择，要么设计更好的算法，要么收集更多的数据。往往做的选择就是设计更好的算法。实际上，收集数据更实用些。</p>
<p>因此这节核心就是有大量数据的笨算法要胜过数据量较少的聪明算法。</p>
<p>对于我们而言，资源有三类：时间 + 内存 + 数据。</p>
<p>在以前，我们会说数据是瓶颈，可是现在都海量数据了。但是，人们似乎觉得没有足够的时间处理海量数据，还是把算法改进到很 perfect 的地步好了。这其实不太对。<br>其实算法越复杂，错误好像也越多。而且，文章说到，采用更聪明的算法也不见得就有好的回报，有一部分原因就是机器学习的机制其实是相同的。不够聪明的算法改到高级也未必使得结果就更好，因为它们有可能有相同的结果。</p>
<p>所以首先尝试最简单的算法总是有好处的。更聪明的算法也更难驾驭。虽然，研究的论文总是在比较结果的准确率和计算复杂度，但是，节省人力和得到知识更重要（这个，的确之前都忽视了）。</p>
<h2 id="Learn-many-models-not-Just-one-要学习很多模型，而不仅仅是一个"><a href="#Learn-many-models-not-Just-one-要学习很多模型，而不仅仅是一个" class="headerlink" title="Learn many models, not Just one 要学习很多模型，而不仅仅是一个"></a>Learn many models, not Just one 要学习很多模型，而不仅仅是一个</h2><p>一般认为一个学习器就够了，但是现在发现多个学习器结合，效果更好。现在有很多模型集成的技术：bagging( 装袋 )、boosting( 强化提升 )、stacking( 堆叠 )。</p>
<h2 id="Simplicity-Does-not-imply-Accuracy-简单并不意味着准确"><a href="#Simplicity-Does-not-imply-Accuracy-简单并不意味着准确" class="headerlink" title="Simplicity Does not imply Accuracy 简单并不意味着准确"></a>Simplicity Does not imply Accuracy 简单并不意味着准确</h2><p>著名的奥坎姆剃刀（occam ’ srazor）原理称：若无必要，勿增实体（entities should not be multiplied beyond necessity）。但是好像被理解成简单的犯错低，就更好。这个其实不对，上面讲到的模型集成，有说会修正分错的例子，所以模型集成的分类器是复杂的，它比简单的要精确。奥坎姆最初的意思是说，开始的时候选择简单的假设，可以修正它，直到效果理想。但是不要一开始从复杂的做起，而不是说要找简单作为最终的学习器。</p>
<h2 id="Representable-Does-not-imply-Learnable-可表示并不意味着可学习"><a href="#Representable-Does-not-imply-Learnable-可表示并不意味着可学习" class="headerlink" title="Representable Does not imply Learnable 可表示并不意味着可学习"></a>Representable Does not imply Learnable 可表示并不意味着可学习</h2><p>一个函数可以被表示出来，不见得就能被学习。这个意思就是说，我们给定了数据、时间和内存，用标准的学习器来学习，但是只能学到所有可能函数中的一部分，并不能学到所有的函数。因此，有的函数它能够写成某种形式，但是，我们也可能没办法求到它。因此，要多试一些学习器。</p>
<h2 id="Correlation-Does-not-imply-Causation-相关并不意味着因果"><a href="#Correlation-Does-not-imply-Causation-相关并不意味着因果" class="headerlink" title="Correlation Does not imply Causation 相关并不意味着因果"></a>Correlation Does not imply Causation 相关并不意味着因果</h2><p>我们只是发现观测变量之间的相关性，但我们还希望从观测数据发现因果信息。另一方面，相关性是因果关系的标志，我们可以将其作为进一步考察的指南（例如试图理解因果链可能是什么样）</p>
<h2 id="LEARNING-REPRESENTATION-EVALUATION-OPTIMIZATION"><a href="#LEARNING-REPRESENTATION-EVALUATION-OPTIMIZATION" class="headerlink" title="LEARNING = REPRESENTATION + EVALUATION + OPTIMIZATION"></a>LEARNING = REPRESENTATION + EVALUATION + OPTIMIZATION</h2><p>Suppose you have an application that you think machine learning might be good for. The first problem facing you is the bewildering variety of learning algorithms available.<br>Which one to use? There are literally thousands available, and hundreds more are published each year. The key to not getting lost in this huge space is to realize that it consists of combinations of just three components. The components are:</p>
<p><strong>Representation</strong><br>A classiddffir must be represented in some formal language that the computer can handle. Conversely, choosing a representation for a learner is tantamount to choosing the set of classiddffirs that it can possibly learn. This set is called the hypothesis space of the learner. If a classiddffir is not in <strong>the hypothesis space</strong>, it cannot be learned. A related question, which we will address in a later section, is how to represent the input, i.e., what features to use.</p>
<p><strong>Evaluation</strong><br>An evaluation function (also called objective function or scoring function) is needed to <strong>distinguish good classiddffirs from bad ones</strong>. The evaluation function used internally by the algorithm may differ from the external one that we want the classifier to optimize, for ease of optimization (see below) and due to the issues discussed in the next section.</p>
<p><strong>Optimization</strong><br>Finally, we need a method to search among the classiddffirs in the language for the highest-scoring one. The choice of optimization technique is key to the efficiency of the learner, and also helps determine the classiddffir produced if the evaluation function has more than one optimum. It is common for new learners to start out using off-the-shelf optimizers, which are later replaced by custom-designed ones.</p>
<p>Table 1 shows common examples of each of these three components.<br>For example,<br>k-nearest neighbor classifies a test example by finding the k most similar training examples and predicting the majority class among them.<br>Hyperplane based methods form a linear combination of the features per class and predict the class with the highest-valued combination.<br>Decision trees test one feature at each internal node, with one branch for each feature value, and have class predictions at the leaves. Algorithm 1 shows a bare-bones decision tree learner for Boolean domains, using information gain and greedy search [21]. InfoGain(xj ,y) is the mutual information between feature xj and the class y. MakeNode(x,c0 ,c1 ) returns a node that tests feature x and has c0 as the child for x = 0 and c1 as the child for x = 1.</p>
<img src="/2017/09/22/A-Few-Useful-Things-to-Know-about-Machine-Learning/markdown-img-paste-20170922163904996.png" alt="Table 1: The three components of learning algorithms." title="Table 1: The three components of learning algorithms.">
<p><em>Important</em><br>Evaluation: Accuracy/Error rate, Precision and recall, Squared error, Likelihood, Posterior probability, Information gain, K-L divergence, Cost/Utility, Margin<br>Optimization: Combinatorial optimization(Greedy search, Beam search, Branch-and-bound); Continuous optimization()</p>
<p>Most textbooks are organized by representation, and it ’ s easy to overlook the fact that the other components are equally important. There is no simple recipe for choosing each component, but the next sections touch on some of the key issues. And, as we will see below, some choices in a machine learning pro ject may be even more important than the choice of learner.</p>
<h2 id="it-’-s-generalization-that-counts"><a href="#it-’-s-generalization-that-counts" class="headerlink" title="it ’ s generalization that counts"></a>it ’ s generalization that counts</h2><p>The fundamental goal of machine learning is to generalize beyond the examples in the training set. This is because, no matter how much data we have, it is very unlikely that we will see those exact examples again at test time. (Notice that, if there are 100,000 words in the dictionary, the spam filter described above has 2100,000 possible different inputs.) Doing well on the training set is easy (just memorize the examples). The most common mistake among machine learning beginners is to test on the training data and have the illusion of success. If the chosen classifier is then tested on new data, it is often no better than random guessing. So, if you hire someone to build a classifier, be sure to keep some of the data to yourself and test the classifier they give you on it. Conversely, if you ’ ve been hired to build a classifier, set some of the data aside from the beginning, and only use it to test your chosen classifier at the very end, followed by learning your final classifier on the whole data.</p>
<h2 id="overfitting-has-many-faces"><a href="#overfitting-has-many-faces" class="headerlink" title="overfitting has many faces"></a>overfitting has many faces</h2><p>One way to understand overfitting is by decomposing generalization error into bias and variance. Bias is a learner ’ s tendency to consistently learn the same wrong thing. Variance is the tendency to learn random things irrespective of the real signal.</p>
<p>It ’ s easy to avoid over ﬁ tting (variance) by falling into the opposite error of under ﬁ tting (bias). Simultaneously avoiding both requires learning a perfect classi ﬁ er, and short of knowing it in advance there is no single technique that will always do best (no free lunch).</p>

      
    </div>
    
    
    

    

    

    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/Paper-Reading/" rel="tag"># Paper Reading</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/09/22/install-keras/" rel="next" title="Install Keras">
                <i class="fa fa-chevron-left"></i> Install Keras
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2017/09/22/Keras-Notes/" rel="prev" title="Keras Notes">
                Keras Notes <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          
  <div class="comments" id="comments">
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="/uploads/avatar.jpg"
               alt="东木金" />
          <p class="site-author-name" itemprop="name">东木金</p>
           
              <p class="site-description motion-element" itemprop="description">正在学习机器学习，希望能变得很强！</p>
          
        </div>
        <nav class="site-state motion-element">

          
            <div class="site-state-item site-state-posts">
              <a href="/archives/">
                <span class="site-state-item-count">162</span>
                <span class="site-state-item-name">日志</span>
              </a>
            </div>
          

          
            
            
            <div class="site-state-item site-state-categories">
              <a href="/categories/index.html">
                <span class="site-state-item-count">18</span>
                <span class="site-state-item-name">分类</span>
              </a>
            </div>
          

          
            
            
            <div class="site-state-item site-state-tags">
              <a href="/tags/index.html">
                <span class="site-state-item-count">42</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/bdmk" target="_blank" title="GitHub">
                  
                    <i class="fa fa-fw fa-github"></i>
                  
                    
                      GitHub
                    
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="mailto:catcherchan94@outlook.com" target="_blank" title="E-Mail">
                  
                    <i class="fa fa-fw fa-envelope"></i>
                  
                    
                      E-Mail
                    
                </a>
              </span>
            
          
        </div>

        
        

        
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#Learning-Representation-Evaluation-optimization-学习-表示-评价-优化"><span class="nav-number">1.</span> <span class="nav-text">Learning = Representation + Evaluation + optimization 学习 = 表示 + 评价 + 优化</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#It-is-Generalization-that-Counts-泛化（Generalization）很重要"><span class="nav-number">2.</span> <span class="nav-text">It is Generalization that Counts 泛化（Generalization）很重要</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Data-Alone-is-not-Enough-仅有数据还不够"><span class="nav-number">3.</span> <span class="nav-text">Data Alone is not Enough 仅有数据还不够</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Overfitting-has-many-Faces-过拟合（Overfitting）有多张面孔"><span class="nav-number">4.</span> <span class="nav-text">Overfitting has many Faces 过拟合（Overfitting）有多张面孔</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Intuition-Fails-in-high-Dimensions-直觉不适用于高维空间"><span class="nav-number">5.</span> <span class="nav-text">Intuition Fails in high Dimensions 直觉不适用于高维空间</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#theoretical-Guarantees-Are-NotWhat-they-Seem-理论保证（Theoretical-Guarantees）与看上去的不一样"><span class="nav-number">6.</span> <span class="nav-text">theoretical Guarantees Are NotWhat they Seem 理论保证（Theoretical  Guarantees）与看上去的不一样</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Feature-Engineering-Is-the-Key-特征工程（Feature-Engineering）是关键"><span class="nav-number">7.</span> <span class="nav-text">Feature Engineering Is the Key 特征工程（Feature Engineering）是关键</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#more-Data-Beats-a-Cleverer-Algorithm-更多是数据胜过更聪明的算法"><span class="nav-number">8.</span> <span class="nav-text">more Data Beats a Cleverer Algorithm 更多是数据胜过更聪明的算法</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Learn-many-models-not-Just-one-要学习很多模型，而不仅仅是一个"><span class="nav-number">9.</span> <span class="nav-text">Learn many models, not Just one 要学习很多模型，而不仅仅是一个</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Simplicity-Does-not-imply-Accuracy-简单并不意味着准确"><span class="nav-number">10.</span> <span class="nav-text">Simplicity Does not imply Accuracy 简单并不意味着准确</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Representable-Does-not-imply-Learnable-可表示并不意味着可学习"><span class="nav-number">11.</span> <span class="nav-text">Representable Does not imply Learnable 可表示并不意味着可学习</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#Correlation-Does-not-imply-Causation-相关并不意味着因果"><span class="nav-number">12.</span> <span class="nav-text">Correlation Does not imply Causation 相关并不意味着因果</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#LEARNING-REPRESENTATION-EVALUATION-OPTIMIZATION"><span class="nav-number">13.</span> <span class="nav-text">LEARNING = REPRESENTATION + EVALUATION + OPTIMIZATION</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#it-’-s-generalization-that-counts"><span class="nav-number">14.</span> <span class="nav-text">it ’ s generalization that counts</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#overfitting-has-many-faces"><span class="nav-number">15.</span> <span class="nav-text">overfitting has many faces</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright" >
  
  &copy;  2017 - 
  <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">东木金</span>
</div>


<div class="powered-by">
  由 <a class="theme-link" href="https://hexo.io">Hexo</a> 强力驱动
</div>

<div class="theme-info">
  主题 -
  <a class="theme-link" href="https://github.com/iissnan/hexo-theme-next">
    NexT.Gemini
  </a>
</div>


        

        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.2"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.2"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=5.1.2"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=5.1.2"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.2"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.2"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.2"></script>



  


  




	





  





  






  

  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    var isXml = true;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length === 0) {
      search_path = "search.xml";
    } else if (/json$/i.test(search_path)) {
      isXml = false;
    }
    var path = "/" + search_path;
    // monitor main search box;

    var onPopupClose = function (e) {
      $('.popup').hide();
      $('#local-search-input').val('');
      $('.search-result-list').remove();
      $('#no-result').remove();
      $(".local-search-pop-overlay").remove();
      $('body').css('overflow', '');
    }

    function proceedsearch() {
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay"></div>')
        .css('overflow', 'hidden');
      $('.search-popup-overlay').click(onPopupClose);
      $('.popup').toggle();
      var $localSearchInput = $('#local-search-input');
      $localSearchInput.attr("autocapitalize", "none");
      $localSearchInput.attr("autocorrect", "off");
      $localSearchInput.focus();
    }

    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';

      // start loading animation
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay">' +
          '<div id="search-loading-icon">' +
          '<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
          '</div>' +
          '</div>')
        .css('overflow', 'hidden');
      $("#search-loading-icon").css('margin', '20% auto 0 auto').css('text-align', 'center');

      $.ajax({
        url: path,
        dataType: isXml ? "xml" : "json",
        async: true,
        success: function(res) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = isXml ? $("entry", res).map(function() {
            return {
              title: $("title", this).text(),
              content: $("content",this).text(),
              url: $("url" , this).text()
            };
          }).get() : res;
          var input = document.getElementById(search_id);
          var resultContent = document.getElementById(content_id);
          var inputEventFunction = function() {
            var searchText = input.value.trim().toLowerCase();
            var keywords = searchText.split(/[\s\-]+/);
            if (keywords.length > 1) {
              keywords.push(searchText);
            }
            var resultItems = [];
            if (searchText.length > 0) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var hitCount = 0;
                var searchTextCount = 0;
                var title = data.title.trim();
                var titleInLowerCase = title.toLowerCase();
                var content = data.content.trim().replace(/<[^>]+>/g,"");
                var contentInLowerCase = content.toLowerCase();
                var articleUrl = decodeURIComponent(data.url);
                var indexOfTitle = [];
                var indexOfContent = [];
                // only match articles with not empty titles
                if(title != '') {
                  keywords.forEach(function(keyword) {
                    function getIndexByWord(word, text, caseSensitive) {
                      var wordLen = word.length;
                      if (wordLen === 0) {
                        return [];
                      }
                      var startPosition = 0, position = [], index = [];
                      if (!caseSensitive) {
                        text = text.toLowerCase();
                        word = word.toLowerCase();
                      }
                      while ((position = text.indexOf(word, startPosition)) > -1) {
                        index.push({position: position, word: word});
                        startPosition = position + wordLen;
                      }
                      return index;
                    }

                    indexOfTitle = indexOfTitle.concat(getIndexByWord(keyword, titleInLowerCase, false));
                    indexOfContent = indexOfContent.concat(getIndexByWord(keyword, contentInLowerCase, false));
                  });
                  if (indexOfTitle.length > 0 || indexOfContent.length > 0) {
                    isMatch = true;
                    hitCount = indexOfTitle.length + indexOfContent.length;
                  }
                }

                // show search results

                if (isMatch) {
                  // sort index by position of keyword

                  [indexOfTitle, indexOfContent].forEach(function (index) {
                    index.sort(function (itemLeft, itemRight) {
                      if (itemRight.position !== itemLeft.position) {
                        return itemRight.position - itemLeft.position;
                      } else {
                        return itemLeft.word.length - itemRight.word.length;
                      }
                    });
                  });

                  // merge hits into slices

                  function mergeIntoSlice(text, start, end, index) {
                    var item = index[index.length - 1];
                    var position = item.position;
                    var word = item.word;
                    var hits = [];
                    var searchTextCountInSlice = 0;
                    while (position + word.length <= end && index.length != 0) {
                      if (word === searchText) {
                        searchTextCountInSlice++;
                      }
                      hits.push({position: position, length: word.length});
                      var wordEnd = position + word.length;

                      // move to next position of hit

                      index.pop();
                      while (index.length != 0) {
                        item = index[index.length - 1];
                        position = item.position;
                        word = item.word;
                        if (wordEnd > position) {
                          index.pop();
                        } else {
                          break;
                        }
                      }
                    }
                    searchTextCount += searchTextCountInSlice;
                    return {
                      hits: hits,
                      start: start,
                      end: end,
                      searchTextCount: searchTextCountInSlice
                    };
                  }

                  var slicesOfTitle = [];
                  if (indexOfTitle.length != 0) {
                    slicesOfTitle.push(mergeIntoSlice(title, 0, title.length, indexOfTitle));
                  }

                  var slicesOfContent = [];
                  while (indexOfContent.length != 0) {
                    var item = indexOfContent[indexOfContent.length - 1];
                    var position = item.position;
                    var word = item.word;
                    // cut out 100 characters
                    var start = position - 20;
                    var end = position + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if (end < position + word.length) {
                      end = position + word.length;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    slicesOfContent.push(mergeIntoSlice(content, start, end, indexOfContent));
                  }

                  // sort slices in content by search text's count and hits' count

                  slicesOfContent.sort(function (sliceLeft, sliceRight) {
                    if (sliceLeft.searchTextCount !== sliceRight.searchTextCount) {
                      return sliceRight.searchTextCount - sliceLeft.searchTextCount;
                    } else if (sliceLeft.hits.length !== sliceRight.hits.length) {
                      return sliceRight.hits.length - sliceLeft.hits.length;
                    } else {
                      return sliceLeft.start - sliceRight.start;
                    }
                  });

                  // select top N slices in content

                  var upperBound = parseInt('1');
                  if (upperBound >= 0) {
                    slicesOfContent = slicesOfContent.slice(0, upperBound);
                  }

                  // highlight title and content

                  function highlightKeyword(text, slice) {
                    var result = '';
                    var prevEnd = slice.start;
                    slice.hits.forEach(function (hit) {
                      result += text.substring(prevEnd, hit.position);
                      var end = hit.position + hit.length;
                      result += '<b class="search-keyword">' + text.substring(hit.position, end) + '</b>';
                      prevEnd = end;
                    });
                    result += text.substring(prevEnd, slice.end);
                    return result;
                  }

                  var resultItem = '';

                  if (slicesOfTitle.length != 0) {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + highlightKeyword(title, slicesOfTitle[0]) + "</a>";
                  } else {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + title + "</a>";
                  }

                  slicesOfContent.forEach(function (slice) {
                    resultItem += "<a href='" + articleUrl + "'>" +
                      "<p class=\"search-result\">" + highlightKeyword(content, slice) +
                      "...</p>" + "</a>";
                  });

                  resultItem += "</li>";
                  resultItems.push({
                    item: resultItem,
                    searchTextCount: searchTextCount,
                    hitCount: hitCount,
                    id: resultItems.length
                  });
                }
              })
            };
            if (keywords.length === 1 && keywords[0] === "") {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>'
            } else if (resultItems.length === 0) {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>'
            } else {
              resultItems.sort(function (resultLeft, resultRight) {
                if (resultLeft.searchTextCount !== resultRight.searchTextCount) {
                  return resultRight.searchTextCount - resultLeft.searchTextCount;
                } else if (resultLeft.hitCount !== resultRight.hitCount) {
                  return resultRight.hitCount - resultLeft.hitCount;
                } else {
                  return resultRight.id - resultLeft.id;
                }
              });
              var searchResultList = '<ul class=\"search-result-list\">';
              resultItems.forEach(function (result) {
                searchResultList += result.item;
              })
              searchResultList += "</ul>";
              resultContent.innerHTML = searchResultList;
            }
          }

          if ('auto' === 'manual') {
            input.addEventListener('input', inputEventFunction);
          } else {
            $('.search-icon').click(inputEventFunction);
            input.addEventListener('keypress', function (event) {
              if (event.keyCode === 13) {
                inputEventFunction();
              }
            });
          }

          // remove loading animation
          $(".local-search-pop-overlay").remove();
          $('body').css('overflow', '');

          proceedsearch();
        }
      });
    }

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched === false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(onPopupClose);
    $('.popup').click(function(e){
      e.stopPropagation();
    });
    $(document).on('keyup', function (event) {
      var shouldDismissSearchPopup = event.which === 27 &&
        $('.search-popup').is(':visible');
      if (shouldDismissSearchPopup) {
        onPopupClose();
      }
    });
  </script>





  

  

  

  
  
    <script type="text/x-mathjax-config">
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [ ['$','$'], ["\\(","\\)"]  ],
          processEscapes: true,
          skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
        }
      });
    </script>

    <script type="text/x-mathjax-config">
      MathJax.Hub.Queue(function() {
        var all = MathJax.Hub.getAllJax(), i;
        for (i=0; i < all.length; i += 1) {
          all[i].SourceElement().parentNode.className += ' has-jax';
        }
      });
    </script>
    <script type="text/javascript" src="//cdn.bootcss.com/mathjax/2.7.1/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
  


  

  

</body>
</html>
